import scrapy
from eolcrawl.spiders.comspider import ComlistSpider
from playwright.async_api import Page
from scrapy_playwright.page import PageMethod
from eolcrawl.spiderutils.common import build_url, format_str
from eolcrawl.items import RosListItem
from eolcrawl.spiders.comspider import set_listspider_feeds

class RosSourceSpider(ComlistSpider):
    name = "ros_source"
    allowed_domains = ["packages.ros.org"]
    start_urls = ["https://packages.ros.org/ros2/ubuntu/pool/main/r"]
    site_name = "ros"
    category = ""
    # is_debug = True

    @classmethod
    def from_crawler(cls, crawler, *args, **kwargs):
        spider = super(RosSourceSpider, cls).from_crawler(crawler, *args, **kwargs)
        set_listspider_feeds(spider,crawler)
        return spider

    def start_requests(self):
            for url in self.start_urls:
                self.logger.debug(f"start_requests:{url}")
                yield scrapy.Request(
                    url,
                    meta={
                        "playwright": True,
                        "playwright_include_page": True,
                        "playwright_context_kwargs": {
                            "ignore_https_errors": True  # 覆盖全局设置
                        }
                        # "playwright_page_methods": [
                        #     PageMethod(self.scroll_page),
                        #     # PageMethod("screenshot", path=self.name+".png", full_page=True),
                        # ],
                    },
                    callback=self.parse,
                )
                
    async def parse(self, response, **kwargs):
        tbody_trs = response.css("tbody tr")
        
        cnt = 0
        for tr in tbody_trs:
            cnt += 1
            if self.is_debug:
                if cnt > 10:
                    break
                
            _item = RosListItem()
            if tr.css('th'):
                continue
            if tr.css('td'):
                href = tr.css("td a").attrib["href"] if tr.css("td a") else None
                href = href.strip() if href else None
                if href is not None and href.startswith("ros-"):
                    name = tr.css("td a::text").get() if tr.css("td a") else None
                    _item["name"] = name.strip()[:-1] if name else None
                    _item['branch'] = name.split("ros-")[1] if name else None
                    _item["url"]  = build_url(response.url,href)
                    
                    date_str = tr.css("td:nth-child(3)::text").get() if tr.css("td:nth-child(3)") else None
                    _item["_date"]=date_str.strip() if date_str else None
                    self.logger.debug(f"parse item:{_item}")
                    
                    yield _item
                    
                else:
                    self.logger.info("Is not find ros-")
            else:  
                self.logger.info("Is no find td tag")
    
            pass
        # for tr in tbody_trs:
        #     _item = RosSourceItem()   
        
        
        
        
        pass
